{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import time\n",
    "import os\n",
    "import re\n",
    "import collections\n",
    "import random\n",
    "import pickle\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.cross_validation import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "maxlen = 20\n",
    "location = os.getcwd()\n",
    "learning_rate = 0.0001\n",
    "batch = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('dataset-emotion.p', 'rb') as fopen:\n",
    "    df = pickle.load(fopen)\n",
    "with open('vector-emotion.p', 'rb') as fopen:\n",
    "    vectors = pickle.load(fopen)\n",
    "with open('dataset-dictionary.p', 'rb') as fopen:\n",
    "    dictionary = pickle.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "label = np.unique(df[:,1])\n",
    "train_X, test_X, train_Y, test_Y = train_test_split(df[:,0], df[:, 1].astype('int'), test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def embed_seq(inputs, vocab_size=None, embed_dim=None, zero_pad=False, scale=False):\n",
    "    lookup_table = tf.get_variable('lookup_table', dtype=tf.float32, shape=[vocab_size, embed_dim])\n",
    "    if zero_pad:\n",
    "        lookup_table = tf.concat((tf.zeros([1, embed_dim]), lookup_table[1:, :]), axis=0)\n",
    "    outputs = tf.nn.embedding_lookup(lookup_table, inputs)\n",
    "    if scale:\n",
    "        outputs = outputs * (embed_dim ** 0.5)\n",
    "    return outputs\n",
    "\n",
    "def learned_positional_encoding(inputs, embed_dim, zero_pad=False, scale=False):\n",
    "    T = inputs.get_shape().as_list()[1]\n",
    "    outputs = tf.range(T) \n",
    "    outputs = tf.expand_dims(outputs, 0)\n",
    "    outputs = tf.tile(outputs, [tf.shape(inputs)[0], 1])\n",
    "    return embed_seq(outputs, T, embed_dim, zero_pad=zero_pad, scale=scale)\n",
    "\n",
    "def layer_norm(inputs, epsilon=1e-8):\n",
    "    mean, variance = tf.nn.moments(inputs, [-1], keep_dims=True)\n",
    "    normalized = (inputs - mean) / (tf.sqrt(variance + epsilon))\n",
    "    params_shape = inputs.get_shape()[-1:]\n",
    "    gamma = tf.get_variable('gamma', params_shape, tf.float32, tf.ones_initializer())\n",
    "    beta = tf.get_variable('beta', params_shape, tf.float32, tf.zeros_initializer())\n",
    "    return gamma * normalized + beta\n",
    "\n",
    "def pointwise_feedforward(inputs, num_units=[None, None], activation=None):\n",
    "    outputs = tf.layers.conv1d(inputs, num_units[0], kernel_size=1, activation=activation)\n",
    "    outputs = tf.layers.conv1d(outputs, num_units[1], kernel_size=1, activation=None)\n",
    "    outputs += inputs\n",
    "    outputs = layer_norm(outputs)\n",
    "    return outputs\n",
    "\n",
    "class Model:\n",
    "    def __init__(self, dimension_input, dimension_output, seq_len,\n",
    "                 learning_rate, num_heads=8, attn_windows=range(1, 6)):\n",
    "        self.size_layer = dimension_input\n",
    "        self.num_heads = num_heads\n",
    "        self.seq_len = seq_len\n",
    "        self.X = tf.placeholder(tf.float32, [None, seq_len, dimension_input])\n",
    "        self.Y = tf.placeholder(tf.float32, [None, dimension_output])\n",
    "        feed = self.X\n",
    "        for i, win_size in enumerate(attn_windows):\n",
    "            with tf.variable_scope('attn_masked_window_%d' % win_size):\n",
    "                feed = self.multihead_attn(feed, self.window_mask(win_size))\n",
    "        feed += learned_positional_encoding(feed, dimension_input)\n",
    "        with tf.variable_scope('multihead'):\n",
    "            feed = self.multihead_attn(feed, None)\n",
    "        with tf.variable_scope('pointwise'):\n",
    "            feed = pointwise_feedforward(feed, num_units=[4*dimension_input, \n",
    "                                                          dimension_input], activation=tf.nn.relu)\n",
    "        self.logits = tf.layers.dense(feed, dimension_output)[:,-1]\n",
    "        self.cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = self.logits, labels = self.Y))\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(self.cost)\n",
    "        self.correct_pred = tf.equal(tf.argmax(self.logits, 1), tf.argmax(self.Y, 1))\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(self.correct_pred, tf.float32))\n",
    "        \n",
    "    def multihead_attn(self, inputs, masks):\n",
    "        T_q = T_k = inputs.get_shape().as_list()[1]           \n",
    "        Q_K_V = tf.layers.dense(inputs, 3*self.size_layer, tf.nn.relu)\n",
    "        Q, K, V = tf.split(Q_K_V, 3, -1)\n",
    "        Q_ = tf.concat(tf.split(Q, self.num_heads, axis=2), axis=0)\n",
    "        K_ = tf.concat(tf.split(K, self.num_heads, axis=2), axis=0)\n",
    "        V_ = tf.concat(tf.split(V, self.num_heads, axis=2), axis=0)\n",
    "        align = tf.matmul(Q_, tf.transpose(K_, [0,2,1]))\n",
    "        align = align / np.sqrt(K_.get_shape().as_list()[-1])\n",
    "        if masks is not None:\n",
    "            paddings = tf.fill(tf.shape(align), float('-inf'))                         \n",
    "            align = tf.where(tf.equal(masks, 0), paddings, align) \n",
    "        align = tf.nn.softmax(align)\n",
    "        outputs = tf.matmul(align, V_)\n",
    "        outputs = tf.concat(tf.split(outputs, self.num_heads, axis=0), axis=2)\n",
    "        outputs += inputs                \n",
    "        return layer_norm(outputs)\n",
    "        \n",
    "    def window_mask(self, h_w):\n",
    "        masks = np.zeros([self.seq_len, self.seq_len])\n",
    "        for i in range(self.seq_len):\n",
    "            if i < h_w:\n",
    "                masks[i, :i+h_w+1] = 1.\n",
    "            elif i > self.seq_len - h_w - 1:\n",
    "                masks[i, i-h_w:] = 1.\n",
    "            else:                                                             \n",
    "                masks[i, i-h_w:i+h_w+1] = 1.\n",
    "        masks = tf.convert_to_tensor(masks)\n",
    "        return tf.tile(tf.expand_dims(masks,0), [tf.shape(self.X)[0]*self.num_heads, 1, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'unwarrentedly'\n",
      "epoch: 0 , pass acc: 0 , current acc: 0.906650661635084\n",
      "time taken: 360.8515956401825\n",
      "epoch: 1 , training loss: 0.3769192298182611 , training acc: 0.856868627225845 , valid loss: 0.2049889931050526 , valid acc: 0.906650661635084\n",
      "'unwarrentedly'\n",
      "epoch: 1 , pass acc: 0.906650661635084 , current acc: 0.9088355347174271\n",
      "time taken: 416.46569561958313\n",
      "epoch: 2 , training loss: 0.18634120830690604 , training acc: 0.9151859655127099 , valid loss: 0.18621045338059244 , valid acc: 0.9088355347174271\n",
      "'unwarrentedly'\n",
      "epoch: 2 , pass acc: 0.9088355347174271 , current acc: 0.9088475400803326\n",
      "time taken: 442.62054228782654\n",
      "epoch: 3 , training loss: 0.16586226459210443 , training acc: 0.9208668282844381 , valid loss: 0.1816691227328806 , valid acc: 0.9088475400803326\n",
      "'unwarrentedly'\n",
      "time taken: 381.1996784210205\n",
      "epoch: 4 , training loss: 0.1508502135221999 , training acc: 0.9254499121418812 , valid loss: 0.1851022613166618 , valid acc: 0.9072148882899107\n",
      "'unwarrentedly'\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model(256, label.shape[0], maxlen, learning_rate)\n",
    "sess.run(tf.global_variables_initializer())\n",
    "dimension = vectors.shape[1]\n",
    "saver = tf.train.Saver(tf.global_variables())\n",
    "EARLY_STOPPING, CURRENT_CHECKPOINT, CURRENT_ACC, EPOCH = 10, 0, 0, 0\n",
    "while True:\n",
    "    lasttime = time.time()\n",
    "    if CURRENT_CHECKPOINT == EARLY_STOPPING:\n",
    "        print('break epoch:', EPOCH)\n",
    "        break\n",
    "    train_acc, train_loss, test_acc, test_loss = 0, 0, 0, 0\n",
    "    for i in range(0, (train_X.shape[0] // batch) * batch, batch):\n",
    "        batch_x = np.zeros((batch, maxlen, 256))\n",
    "        batch_y = np.zeros((batch, len(label)))\n",
    "        for k in range(batch):\n",
    "            tokens = train_X[i + k].split()[:maxlen]\n",
    "            emb_data = np.zeros((maxlen, 256), dtype = np.float32)\n",
    "            for no, text in enumerate(tokens[::-1]):\n",
    "                try:\n",
    "                    emb_data[-1 - no, :] += vectors[dictionary[text], :256]\n",
    "                except Exception as e:\n",
    "                    print(e)\n",
    "                    continue\n",
    "            batch_y[k, int(train_Y[i + k])] = 1.0\n",
    "            batch_x[k, :, :] = emb_data[:, :]\n",
    "        loss, _ = sess.run([model.cost, model.optimizer], feed_dict = {model.X : batch_x, model.Y : batch_y})\n",
    "        train_loss += loss\n",
    "        train_acc += sess.run(model.accuracy, feed_dict = {model.X : batch_x, model.Y : batch_y})\n",
    "    \n",
    "    for i in range(0, (test_X.shape[0] // batch) * batch, batch):\n",
    "        batch_x = np.zeros((batch, maxlen, 256))\n",
    "        batch_y = np.zeros((batch, len(label)))\n",
    "        for k in range(batch):\n",
    "            tokens = test_X[i + k].split()[:maxlen]\n",
    "            emb_data = np.zeros((maxlen, 256), dtype = np.float32)\n",
    "            for no, text in enumerate(tokens[::-1]):\n",
    "                try:\n",
    "                    emb_data[-1 - no, :] += vectors[dictionary[text], :256]\n",
    "                except:\n",
    "                    continue\n",
    "            batch_y[k, int(test_Y[i + k])] = 1.0\n",
    "            batch_x[k, :, :] = emb_data[:, :]\n",
    "        loss, acc = sess.run([model.cost, model.accuracy], feed_dict = {model.X : batch_x, model.Y : batch_y})\n",
    "        test_loss += loss\n",
    "        test_acc += acc\n",
    "        \n",
    "    train_loss /= (train_X.shape[0] // batch)\n",
    "    train_acc /= (train_X.shape[0] // batch)\n",
    "    test_loss /= (test_X.shape[0] // batch)\n",
    "    test_acc /= (test_X.shape[0] // batch)\n",
    "    if test_acc > CURRENT_ACC:\n",
    "        print('epoch:', EPOCH, ', pass acc:', CURRENT_ACC, ', current acc:', test_acc)\n",
    "        CURRENT_ACC = test_acc\n",
    "        CURRENT_CHECKPOINT = 0\n",
    "        saver.save(sess, os.getcwd() + \"/model-rnn-vector.ckpt\")\n",
    "    else:\n",
    "        CURRENT_CHECKPOINT += 1\n",
    "    EPOCH += 1\n",
    "    print('time taken:', time.time()-lasttime)\n",
    "    print('epoch:', EPOCH, ', training loss:', train_loss, ', training acc:', train_acc, ', valid loss:', test_loss, ', valid acc:', test_acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
